package com.news.controller;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.news.common.*;
import com.news.entity.SysNews;
import com.news.service.NewsCategoryService;
import com.news.service.SysNewsService;
import com.news.service.SysUserService;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;

import javax.servlet.http.HttpSession;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * #娱乐明星  4-20074
 *
 * #娱乐电影  4-20075
 * #娱乐音乐  4-20077
 *
 *
 */

@Controller
@RequestMapping("grap")
public class GrapNewsController {

    @Autowired
    SysNewsService newsService;

    @Autowired
    SysUserService userService;

    @Autowired
    NewsCategoryService categoryService;

    private static  final Logger logger= LoggerFactory.getLogger(GrapNewsController.class);

    //设置爬取网页
    private static  String baseUrlStar="https://ent.ifeng.com/star/";//娱乐·明星首页
    private static  String baseUrlMovie="https://ent.ifeng.com/movie/";//娱乐·电影首页
    private static  String baseUrlMusic="https://ent.ifeng.com/music/";//娱乐·音乐首页

    //爬取文件存放路径
    //
    @Value("${com.cgx.file.baseFilePath}")
    private String baseFilePath;

    private static String yuleyiyueUrl="https://shankapi.ifeng.com/shanklist/_/getColumnInfo/_/default/{id}/{idTimeMills}/20/4-20077-/getColumnInfoCallback?callback=getColumnInfoCallback&_={currTimeMills}";
    private static String yulestarUrl="https://shankapi.ifeng.com/shanklist/_/getColumnInfo/_/default/{id}/{idTimeMills}/20/4-20074-/getColumnInfoCallback?callback=getColumnInfoCallback&_={currTimeMills}";
    private static String yulefilmUrl="https://shankapi.ifeng.com/shanklist/_/getColumnInfo/_/default/{id}/{idTimeMills}/20/4-20075-/getColumnInfoCallback?callback=getColumnInfoCallback&_={currTimeMills}";

    private String defaultYuLeId="6760584714211311643";//娱乐音乐
    private String defaultYuLeTimeMills="1611846336000";//娱乐音乐

    private String defaultYuLeStarId="6776520715089744701";//娱乐明星
    private String defaultYuLeStarTimeMills="1615739836000";//娱乐明星

    private String defaultYuLeFilmId="6776324094754296302";//娱乐电影
    private String defaultYuLeFilmTimeMills="1615604038000";//娱乐电影

    //爬取凤凰新闻 爬取条数可不输入，默认20

    @RequestMapping("/grapYuLeNews")
    @ResponseBody
    public ResponseBean reptileFile(HttpSession session, @RequestParam(name="number",defaultValue = "20",required = false) Integer number, Long categoryId) {
        String categoryName=categoryService.getById(categoryId).getName();
        AtomicInteger ait=new AtomicInteger(0);
        Date date = new Date();
        Date date1 = DateUtils.addDay(date, 168311);
        logger.debug("当前时间戳：{}", System.currentTimeMillis());
        long timeMillis = DateUtils.getTimeMillis(date1);
        logger.debug("转换后的时间戳：{}", timeMillis);
        JSONObject contentObj = new JSONObject();
        String grapUrl=null;
        switch (categoryName){
            case "娱乐·音乐":
                grapUrl=yuleyiyueUrl;
                JSONObject firstIdAndTimeMills = getFirstIdAndTimeMills(baseUrlMusic);
                defaultYuLeId = firstIdAndTimeMills.getString("id");//新闻ID
                defaultYuLeTimeMills = firstIdAndTimeMills.getString("idTimeMills");//新闻发布时间
                contentObj.put("id", defaultYuLeId);
                contentObj.put("idTimeMills", defaultYuLeTimeMills);
                break;
            case "娱乐·明星":
                grapUrl=yulestarUrl;
                firstIdAndTimeMills = getFirstIdAndTimeMills(baseUrlStar);
                defaultYuLeStarId = firstIdAndTimeMills.getString("id");//新闻ID
                defaultYuLeStarTimeMills = firstIdAndTimeMills.getString("idTimeMills");//新闻发布时间
                contentObj.put("id", defaultYuLeStarId);
                contentObj.put("idTimeMills", defaultYuLeStarTimeMills);
                break;
            case "娱乐·电影":
                grapUrl=yulefilmUrl;
                firstIdAndTimeMills = getFirstIdAndTimeMills(baseUrlMovie);
                defaultYuLeFilmId = firstIdAndTimeMills.getString("id");//新闻ID
                defaultYuLeFilmTimeMills = firstIdAndTimeMills.getString("idTimeMills");//新闻发布时间
                contentObj.put("id", defaultYuLeFilmId);
                contentObj.put("idTimeMills", defaultYuLeFilmTimeMills);
                break;
        }
        contentObj.put("currTimeMills", timeMillis);
        //解析URL
        String parseUrl = getContent(grapUrl, contentObj);
        logger.debug("第一次获取的grapUrl:{}",parseUrl);
        for(int i=0;i<1000;i++){
            Integer data = getPageData(session, categoryId, ait,number,parseUrl);
            logger.debug("当前抓取数据量：{}",data);
            if(data==-1){
                return ResultUtil.error();
            }
            if(data<number&&data!=999999){
                QueryWrapper<SysNews> queryWrapper = new QueryWrapper<>();
                queryWrapper.eq("category_name", categoryName);
                queryWrapper.orderByDesc("id");
                grapUrl="";
                List<SysNews> checkList = newsService.list(queryWrapper);
                grapUrl=transferUrlParam(checkList,categoryName,contentObj,grapUrl);
                contentObj.put("currTimeMills", System.currentTimeMillis());
                //解析URL
                parseUrl = getContent(grapUrl, contentObj);
            }else{
                break;
            }
        }
        return ResultUtil.success();

    }

//    解析url
    private Integer getPageData(HttpSession session,Long categoryId,AtomicInteger ait,Integer number,String parseUrl) {
        String categoryName=categoryService.getById(categoryId).getName();
        logger.debug("开始抓取的URL：{}", parseUrl);
        String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36";
        Map<String, String> map = new HashMap<>();
        map.put("User-Agent", userAgent);
        String sendGet = HttpUtils.httpGet(parseUrl, map);
        String regex = "getColumnInfoCallback\\(";
        String replaceAll = sendGet.replaceAll(regex, "");
        String substring = replaceAll.substring(0, replaceAll.length() - 1);

        JSONObject parseObject = JSONObject.parseObject(substring);
        if (!parseObject.getString("code").equals("0")) {
            logger.debug("爬取数据失败！");
            return -1;
        }
        String isEnd = parseObject.getJSONObject("data").getString("isEnd");
        if(isEnd.equals("true")){
            logger.debug("已经到末尾，不再抓取~");
            return -2;
        }
        JSONArray newsstreamArray = parseObject.getJSONObject("data").getJSONArray("newsstream");
        for (int i = 0; i < newsstreamArray.size(); i++) {
            JSONObject currObj = newsstreamArray.getJSONObject(i);
            SysNews sysNews = new SysNews();
            sysNews.setCategoryName(categoryName);
            sysNews.setCategoryId(categoryId);
            sysNews.setUserId(ToolsUtils.getLoginUserId(session));
            sysNews.setUserName(ToolsUtils.getLoginUserName(session));
            sysNews.setField9(currObj.toJSONString());
            String id = currObj.getString("id");
            logger.debug("id:{}", id);
            sysNews.setField2(id);//newsId--field2
            String title = currObj.getString("title");
            logger.debug("title:{}", title);
            QueryWrapper<SysNews> queryWrapper=new QueryWrapper<>();
            queryWrapper.eq("title",title);
            List<SysNews> checkListExist = newsService.list(queryWrapper);
            if(ToolsUtils.isNotEmpty(checkListExist)){
                logger.debug("当前新闻{}已经存在，将跳过",title);
                continue;
            }
//
            sysNews.setTitle(title);
            sysNews.setTheme(title);

            String url = currObj.getString("url");
            logger.debug("url:{}", url);//详情页url
            String newsTime = currObj.getString("newsTime");
            logger.debug("newsTime:{}", newsTime);
            sysNews.setField3(newsTime);//新闻发布时间
            String source = currObj.getString("source");
            logger.debug("source:{}", source);
            String commentUrl = currObj.getString("commentUrl");
            logger.debug("commentUrl:{}", commentUrl);
            JSONObject thumbnails = currObj.getJSONObject("thumbnails");//处理图片
            JSONArray imageArray = thumbnails.getJSONArray("image");
            for (int j = 0; j < imageArray.size(); j++) {
                JSONObject imgObj = imageArray.getJSONObject(j);
                String imgUrl = imgObj.getString("url");
                logger.debug("imgUrl:{}", imgUrl);
                String saveName = saveFileByUrl(imgUrl);//抓取图片到本地
//                保存的图片
                sysNews.setField1(saveName);
            }
            sysNews = getDetail(sysNews, url);
            logger.debug("封装好的当前新闻对象：{}", sysNews);
            if(sysNews!=null){
                //保存爬取的新闻
                newsService.save(sysNews);
                ait.getAndIncrement();//计数器+1
                if (ait.get()>=number){
                    break;
                }
            }

            logger.debug("-----------------------------------------------------");
        }

        return ait.get();
    }

    /**
     * 根据不同类目，解析不同的grapUrl   defaultYuLeId  defaultYuLeTimeMills
     * @param checkList
     * @param categoryName
     * @param contentObj
     * @param grapUrl
     * @return
     */
    private String transferUrlParam(List<SysNews> checkList,String categoryName,JSONObject contentObj,String grapUrl){
        if (ToolsUtils.isNotEmpty(checkList)) {
            SysNews lastNews = checkList.get(0);
            switch (categoryName){
                case "娱乐·音乐":
                    grapUrl=yuleyiyueUrl;
                    defaultYuLeId = lastNews.getField2();//新闻ID
                    defaultYuLeTimeMills = DateUtils.getTimeMillis(DateUtils.parseDate(lastNews.getField3())) + "";//新闻发布时间
                    contentObj.put("id", defaultYuLeId);
                    contentObj.put("idTimeMills", defaultYuLeTimeMills);
                    break;
                case "娱乐·明星":
                    grapUrl=yulestarUrl;
                    defaultYuLeStarId = lastNews.getField2();//新闻ID
                    defaultYuLeStarTimeMills = DateUtils.getTimeMillis(DateUtils.parseDate(lastNews.getField3())) + "";//新闻发布时间
                    contentObj.put("id", defaultYuLeStarId);
                    contentObj.put("idTimeMills", defaultYuLeStarTimeMills);
                    break;
                case "娱乐·电影":
                    grapUrl=yulefilmUrl;
                    defaultYuLeFilmId = lastNews.getField2();//新闻ID
                    defaultYuLeFilmTimeMills = DateUtils.getTimeMillis(DateUtils.parseDate(lastNews.getField3())) + "";//新闻发布时间
                    contentObj.put("id", defaultYuLeFilmId);
                    contentObj.put("idTimeMills", defaultYuLeFilmTimeMills);
                    break;
            }
        }else{
            switch (categoryName){
                case "娱乐·音乐":
                    grapUrl=yuleyiyueUrl;
                    contentObj.put("id", defaultYuLeId);
                    contentObj.put("idTimeMills", defaultYuLeTimeMills);
                    break;
                case "娱乐·明星":
                    grapUrl=yulestarUrl;
                    contentObj.put("id", defaultYuLeStarId);
                    contentObj.put("idTimeMills", defaultYuLeStarTimeMills);
                    break;
                case "娱乐·电影":
                    grapUrl=yulefilmUrl;
                    contentObj.put("id", defaultYuLeFilmId);
                    contentObj.put("idTimeMills", defaultYuLeFilmTimeMills);
                    break;
            }
        }
        return grapUrl;
    }

    private static SysNews getDetail(SysNews news, String url) {
        Connection conn = Jsoup.connect(url);
        String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36";
        // 修改http包中的header,伪装成浏览器进行抓取
        conn.header("User-Agent", userAgent);
        Document doc = null;
        try {
            doc = conn.get();
        } catch (IOException e) {
            e.printStackTrace();
        }
        Elements elementsTop = doc.getElementsByClass("artical-25JfwmD5");
        if(elementsTop.size()<1){
             news = tuwenParse(news, doc);
             return null;
        }
        Element first = elementsTop.first();
        Elements elementsByClass = first.getElementsByClass("topic-z1ycr_72");
        String text = elementsByClass.text();
        logger.debug("文章标题：{}",text);
//        设置新闻题目
        news.setTitle(text);
        news.setTheme(text);
        Elements timeList = first.getElementsByClass("time-1Mgp9W-1");
        Element child = timeList.first().child(0);
        logger.debug("文章日期：{}",child.text());
        Element child3 = timeList.first().child(3);
        logger.debug("文章来源：{}",child3.text());
        news.setField4(child3.text());
        Elements mainContent = first.getElementsByClass("main_content-r5RGqegj");
        String content = mainContent.html();
        logger.debug("文章详情：{}",content);
        news.setContent(content);
        return news;
    }

    /**
     * 暂时跳过这种类型的新闻
     * 尝试另外一种详情解析：图片轮播+文字
     */

    private static SysNews tuwenParse(SysNews sysNews,Document document){
        Elements elementsTop= document.getElementsByClass("titL-1ZkkRt8v");
        if(!elementsTop.isEmpty()){
            Element child0 = elementsTop.first().child(0);
            sysNews.setTitle(child0.text());
            Element child1 = elementsTop.first().child(1);
            Element child10 = child1.child(0);
            sysNews.setField3(child10.text());
            sysNews.setField4(child1.child(1).text());//新闻来源
        }
        Elements maincontent = document.getElementsByClass("container-2kKu5p9H");
        sysNews.setContent(maincontent.html());
        return  sysNews;
    }

    //返回第一个Id和时间
    private JSONObject getFirstIdAndTimeMills(String url){
        Connection conn = Jsoup.connect(url);
        String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36";
        // 修改http包中的header,伪装成浏览器进行抓取
        conn.header("User-Agent", userAgent);
        Document doc = null;
        try {
            doc = conn.get();
        } catch (IOException e) {
            e.printStackTrace();
        }

        Elements scriptList = doc.getElementsByTag("script");
        for (Element element : scriptList) {
            /*取得JS变量数组*/
            String[] data = element.data().toString().split("var");
            /*取得单个JS变量*/
            for (String variable : data) {
                /*过滤variable为空的数据*/
                if (variable.contains("=")) {

                    /*取到满足条件的JS变量*/
                    if (variable.contains("allData")) {
                        String[] kvp = variable.split("allData");
                        String dataVal = kvp[1];
                        String trimData = dataVal.trim();
                        String substring = trimData.substring(2, trimData.length() - 1);
                        logger.debug(substring);
                        JSONObject parseObject = JSONObject.parseObject(substring);
                        JSONObject firstObj = parseObject.getJSONArray("newsstream").getJSONObject(0);
                        logger.debug("最新的消息为：{}",firstObj);
                        String id = firstObj.getString("id");
                        Date newsTime = firstObj.getDate("newsTime");
                        JSONObject object=new JSONObject();
                        object.put("id",id);
                        object.put("idTimeMills",DateUtils.getTimeMillis(newsTime));
                        return object;

                    }
                }
            }
        }
        return null;
    }

    //获取网络图片保存到本地
    private String  saveFileByUrl(String imgUrl)  {
        FileOutputStream out = null;
        BufferedInputStream in = null;
        HttpURLConnection connection = null;
        byte[] buf = new byte[1024];
        int len = 0;
        String saveFileName= UUID.randomUUID().toString().replaceAll("-","")+".jpg";
        try {
            URL url = new URL(imgUrl);
            connection = (HttpURLConnection)url.openConnection();
            connection.connect();
            in = new BufferedInputStream(connection.getInputStream());
            out = new FileOutputStream(baseFilePath+ File.separator+saveFileName);
            while ((len = in.read(buf)) != -1) {
                out.write(buf, 0, len);
            }
            out.flush();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                in.close();
                out.close();
                connection.disconnect();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return  saveFileName;
    }

    public static String getContent(String tempalte, JSONObject param)  {
        String regex = "(\\{([_a-zA-Z0-9]+)\\})";
        Pattern p = Pattern.compile(regex);
        Matcher m = p.matcher(tempalte);
        StringBuffer content = new StringBuffer();
        while (m.find()) {
            System.out.println(m.group(1));
            String key = m.group(2);
            String value = "";
            if (param.containsKey(key)) {
                value = param.getString(key);
            } else {
                logger.debug("缺少key{}对应的值",key);
                continue;
            }
            m.appendReplacement(content, value);
        }
        m.appendTail(content);
        return content.toString();
    }


}

/**
 {
 "id": "6770174400034509711",
 "title": "新歌歌词被指不尊重科比，米克•米尔私下向瓦妮莎道歉",
 "url": "https://ent.ifeng.com/c/847TAu2Jlst",
 "skey": "dd25e3",
 "newsTime": "2021-02-24 11:18:45",
 "source": "",
 "type": "article",
 "commentUrl": "ucms_847TAu2Jlst",
 "thumbnails": {
 "image": [
 {
 "width": "230",
 "height": "152",
 "url": "https://x0.ifengimg.com/ucms/2021_09/C6177B086368D3494B1AF436663B4BA719AF74D9_size73_w230_h152.png",
 "kbs": "",
 "small": {
 "_proto": {
 "_classname": "int32"
 },
 "_bValue": 0,
 "value": [

 ],
 "_classname": "List<int32>"
 },
 "big": {
 "_proto": {
 "_classname": "int32"
 },
 "_bValue": 0,
 "value": [

 ],
 "_classname": "List<int32>"
 },
 "_classname": "Tarsapi.Image"
 }
 ],
 "_classname": "Tarsapi.Thumbnails"
 },
 "thumbnailsCount": 1,
 "videoCount": 0,
 "summary": "",
 "searchPath": "4-20074-,4-20077-",
 "base62Id": "847TAu2Jlst"
 }
 */
